import requests
import os
from datetime import datetime
import json
import gzip
#import talib
import html5lib
import re
import StringIO
from pyquery import PyQuery as pq
import traceback

import numpy as np

proxies = {
    "http": "http://192.168.47.1:8580",
    "https": "http://192.168.47.1:8580",
}

def readdata(pagecount=1000):
    res=requests.get("https://stooq.com/q/d/l/?s=xauusd&i=d",proxies=proxies)
    if(res.status_code==200):
        data=[line.strip().split(",") for line in res.text.split("\n")]
        titles=data[0]
        data=data[1:]

        datalist=[]
        befortime=datetime(1980,1,1)
        for line in data:
            if len(line)<5:
                continue
            date=datetime.strptime(line[0],"%Y-%m-%d")
            if date<befortime:
                continue
            datalist.append({
                "Date":line[0],
                "Open":float(line[1]),
                "High":float(line[2]),
                "Low":float(line[3]),
                "Close":float(line[4]),
            })
        return datalist

def GetNew():
    def remove_control_characters(html):
        def str_to_int(s, default, base=10):
            if int(s, base) < 0x10000:
                return unichr(int(s, base))
            return default

        html = re.sub(ur"&#(\d+);?", lambda c: str_to_int(c.group(1), c.group(0)), html)
        html = re.sub(ur"&#[xX]([0-9a-fA-F]+);?", lambda c: str_to_int(c.group(1), c.group(0), base=16), html)
        html = re.sub(ur"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", html)
        return html

    parser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("lxml"),namespaceHTMLElements=False)
    r = requests.get("https://stooq.com/q/?s=xauusd",proxies=proxies)
    try:
        try:
            doc = parser.parse(r.content)
        except:
            match = re.search(r"<meta.*content=\".*charset=(?P<code>[^\";]*)\".*/>", r.content)
            if match:
                code=match.group("code")
                b=r.content.decode(code,errors='ignore')
            b=remove_control_characters(b)
            doc=parser.parse(StringIO(b))
    except Exception,e:
        print e
        traceback.print_exc()
        return None
    doc=pq(doc.getroot())

    close = doc("#aq_xauusd_a2").text()
    high = doc("#aq_xauusd_h").text()
    low = doc("#aq_xauusd_l").text()
    open = doc("#aq_xauusd_o").text()
    date = doc("#aq_xauusd_d2").text()
    return {
        "Date":date,
        "Open":float(open),
        "High":float(high),
        "Low":float(low),
        "Close":float(close),
    }

def AddDI(datalist):
    for one in datalist:
        one["DI"]=(one["Close"]*2+one["Low"]+one["High"])/4
'''
def AddWMA(datalist):
    dis=np.array([one["DI"] for one in datalist])
    WMA = talib.MA(dis,30,matype=1)
    for i in xrange(len(datalist)):
        datalist[i]["wma"]=WMA[i]
def AddMACD(datalist):
    dis=np.array([one["DI"] for one in datalist])
    macd, macdsignal, macdhist=talib.MACDEXT(dis, fastperiod=12, fastmatype=1, slowperiod=26, slowmatype=1, signalperiod=9, signalmatype=1)
    for i in xrange(len(datalist)):
        datalist[i]["macd"]=macd[i],macdsignal[i],macdhist[i]
def AddBBands(datalist):
    dis=np.array([one["DI"] for one in datalist])
    upperband, middleband, lowerband = talib.BBANDS(dis, timeperiod=15, nbdevup=2, nbdevdn=2, matype=1)
    for i in xrange(len(datalist)):
        datalist[i]["bband"]=upperband[i], middleband[i], lowerband[i]
def AddRSI(datalist):
    dis=np.array([one["DI"] for one in datalist])
    res=talib.RSI(dis,14)
    for i in xrange(len(datalist)):
        datalist[i]["rsi"]=res[i]
def AddSTOCH(datalist):
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    close=np.array([one["Close"] for one in datalist])
    slowk, slowd = talib.STOCH(high, low, close, fastk_period=5, slowk_period=3, slowk_matype=1, slowd_period=3, slowd_matype=1)
    for i in xrange(len(datalist)):
        datalist[i]["stoch"]=slowk[i], slowd[i]
def AddSAR(datalist):
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    optInAcceleration = .02
    optInMaximum = .20
    SAR = talib.SAR(high, low, optInAcceleration, optInMaximum)
    for i in xrange(len(datalist)):
        datalist[i]["sar"]=SAR[i]
'''

def savedata(data,dbfile):
    try:
        os.remove(dbfile)
    except:
        pass
    with gzip.GzipFile(dbfile,"w") as f:
        json.dump(data,f)
def loaddate(dbfile):
    with gzip.GzipFile(dbfile,"r") as f:
        return json.load(f)
if __name__ == "__main__":
    datalist=readdata()
    datalist.append(GetNew())
    #datalist=loaddate("data/golddata.gz")
    AddDI(datalist)
    AddWMA(datalist)
    AddMACD(datalist)
    AddBBands(datalist)
    AddRSI(datalist)
    AddSTOCH(datalist)
    AddSAR(datalist)
    print json.dumps(datalist,indent=2)
    savedata(datalist,"data/golddata.gz")
